MNIST indepth analysis



In [1]:

    
#importing functions 
import numpy as np
import matplotlib.pyplot as plt 
import tensorflow as tf
from sklearn.metrics import confusion_matrix
from skimage.util.montage import montage2d
import tflearn
from PIL import Image
%matplotlib inline









    



/Users/Enkay/anaconda/lib/python2.7/site-packages/pandas/computation/__init__.py:19: UserWarning: The installed version of numexpr 2.4.4 is not supported in pandas and will be not be used

  UserWarning)



In [2]:

    
#importing data
from tensorflow.examples.tutorials.mnist import input_data
#one hot encoding returns an array of zeros and a single one. One corresponds to the class
data = input_data.read_data_sets("data/MNIST/", one_hot=True)









    



Extracting data/MNIST/train-images-idx3-ubyte.gz
Extracting data/MNIST/train-labels-idx1-ubyte.gz
Extracting data/MNIST/t10k-images-idx3-ubyte.gz
Extracting data/MNIST/t10k-labels-idx1-ubyte.gz



In [35]:

    
print "Shape of images in training dataset {}".format(data.train.images.shape)
print "Shape of classes in training dataset {}".format(data.train.labels.shape)
print "Shape of images in testing dataset {}".format(data.test.images.shape)
print "Shape of classes in testing dataset {}".format(data.test.labels.shape)
print "Shape of images in validation dataset {}".format(data.validation.images.shape)
print "Shape of classes in validation dataset {}".format(data.validation.labels.shape)









    



Shape of images in training dataset (55000, 784)
Shape of classes in training dataset (55000, 10)
Shape of images in testing dataset (10000, 784)
Shape of classes in testing dataset (10000, 10)
Shape of images in validation dataset (5000, 784)
Shape of classes in validation dataset (5000, 10)



In [4]:

    
#sample image
sample=data.train.images[5].reshape(28,28) 
plt.imshow(sample ,cmap='gray')
plt.title('Sample image')
plt.axis('off')
plt.show()



In [5]:

    
# function to display montage of input data 
imgs=data.train.images[0:100]
montage_img=np.zeros([100,28,28])
for i in range(len(imgs)) : 
        montage_img[i]=imgs[i].reshape(28,28) 
plt.imshow(montage2d(montage_img), cmap='gray')
plt.title('Sample of input data')
plt.axis('off')
plt.show()



In [6]:

    
images=data.train.images
images=np.reshape(images,[images.shape[0],28,28])
mean_img = np.mean(images, axis=0)
std_img = np.std(images, axis=0)



In [7]:

    
plt.imshow(mean_img)
plt.title('Mean image of the data')
plt.colorbar()
plt.axis('off')
plt.show()









    



/Users/Enkay/anaconda/lib/python2.7/site-packages/matplotlib/collections.py:590: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison
  if self._edgecolors == str('face'):



In [8]:

    
plt.imshow(std_img)
plt.colorbar()
plt.title('Standard deviation of the data')
plt.axis('off')
plt.show()



In [9]:

    
#input - shape 'None' states that, the value can be anything, i.e we can feed in any number of images
#input image
x=tf.placeholder(tf.float32,shape=[None,784]) 
#input class
y_=tf.placeholder(tf.float32,shape=[None, 10])

Our Model:

Series of convolutional layers followed by fullyconnected layer and a softmax layer.

Convolutional layer:

Each convolutional layer consists of convolution operation followed by nonlinear activation function and pooling layer.

Our model layout:

Input layer --> Convolutional layer 1 --> Convolutional layer 2 --> Fully Connected Layer -- >Softmax layer



In [10]:

    
#Input Layer

#reshaping input for convolutional operation in tensorflow
# '-1' states that there is no fixed batch dimension, 28x28(=784) is reshaped from 784 pixels and '1' for a single
#channel, i.e a gray scale image
x_input=tf.reshape(x,[-1,28,28,1], name='input')
#first convolutional layer with 32 output filters, filter size 5x5, stride of 2,same padding, and RELU activation.
#please note, I am not adding bias, but one could add bias.Optionally you can add max pooling layer as well 
conv_layer1=tflearn.layers.conv.conv_2d(x_input, nb_filter=32, filter_size=5, strides=[1,1,1,1],
                                        padding='same', activation='relu', regularizer="L2", name='conv_layer_1')

#2x2 max pooling layer
out_layer1=tflearn.layers.conv.max_pool_2d(conv_layer1, 2)


#second convolutional layer 
conv_layer2=tflearn.layers.conv.conv_2d(out_layer1, nb_filter=32, filter_size=5, strides=[1,1,1,1],
                                        padding='same', activation='relu',  regularizer="L2", name='conv_layer_2')
out_layer2=tflearn.layers.conv.max_pool_2d(conv_layer2, 2)
#fully connected layer
fcl= tflearn.layers.core.fully_connected(out_layer2, 1024, activation='relu')
fcl_dropout = tflearn.layers.core.dropout(fcl, 0.8)
y_predicted = tflearn.layers.core.fully_connected(fcl_dropout, 10, activation='softmax', name='output')



In [11]:

    
print "Shape of input : {}".format(x_input.get_shape().as_list())
print "Shape of first convolutional layer : {}".format(out_layer1.get_shape().as_list())
print "Shape of second convolutional layer : {}".format(out_layer2.get_shape().as_list())
print "Shape of fully connected layer : {}".format(fcl.get_shape().as_list())
print "Shape of output layer : {}".format(y_predicted.get_shape().as_list())









    



Shape of input : [None, 28, 28, 1]
Shape of first convolutional layer : [None, 14, 14, 32]
Shape of second convolutional layer : [None, 7, 7, 32]
Shape of fully connected layer : [None, 1024]
Shape of output layer : [None, 10]



In [12]:

    
#loss function
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_predicted), reduction_indices=[1]))
#optimiser -
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
#calculating accuracy of our model 
correct_prediction = tf.equal(tf.argmax(y_predicted,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))



In [13]:

    
#session parameters
sess = tf.InteractiveSession()
#initialising variables
init = tf.initialize_all_variables()
sess.run(init)



In [14]:

    
# grabbing the default graph
g = tf.get_default_graph()

# every operations in our graph
[op.name for op in g.get_operations()]









    Out[14]:





[u'Placeholder',
 u'Placeholder_1',
 u'input/shape',
 u'input',
 u'conv_layer_1/W',
 u'conv_layer_1/W/Initializer/random_uniform/shape',
 u'conv_layer_1/W/Initializer/random_uniform/min',
 u'conv_layer_1/W/Initializer/random_uniform/max',
 u'conv_layer_1/W/Initializer/random_uniform/RandomUniform',
 u'conv_layer_1/W/Initializer/random_uniform/sub',
 u'conv_layer_1/W/Initializer/random_uniform/mul',
 u'conv_layer_1/W/Initializer/random_uniform',
 u'conv_layer_1/W/Assign',
 u'conv_layer_1/W/read',
 u'conv_layer_1/W/Regularizer/L2Loss',
 u'conv_layer_1/W/Regularizer/L2-Loss/y',
 u'conv_layer_1/W/Regularizer/L2-Loss',
 u'conv_layer_1/b',
 u'conv_layer_1/b/Initializer/Const',
 u'conv_layer_1/b/Assign',
 u'conv_layer_1/b/read',
 u'conv_layer_1/Conv2D',
 u'conv_layer_1/BiasAdd',
 u'conv_layer_1/Relu',
 u'MaxPool2D/MaxPool',
 u'conv_layer_2/W',
 u'conv_layer_2/W/Initializer/random_uniform/shape',
 u'conv_layer_2/W/Initializer/random_uniform/min',
 u'conv_layer_2/W/Initializer/random_uniform/max',
 u'conv_layer_2/W/Initializer/random_uniform/RandomUniform',
 u'conv_layer_2/W/Initializer/random_uniform/sub',
 u'conv_layer_2/W/Initializer/random_uniform/mul',
 u'conv_layer_2/W/Initializer/random_uniform',
 u'conv_layer_2/W/Assign',
 u'conv_layer_2/W/read',
 u'conv_layer_2/W/Regularizer/L2Loss',
 u'conv_layer_2/W/Regularizer/L2-Loss/y',
 u'conv_layer_2/W/Regularizer/L2-Loss',
 u'conv_layer_2/b',
 u'conv_layer_2/b/Initializer/Const',
 u'conv_layer_2/b/Assign',
 u'conv_layer_2/b/read',
 u'conv_layer_2/Conv2D',
 u'conv_layer_2/BiasAdd',
 u'conv_layer_2/Relu',
 u'MaxPool2D_1/MaxPool',
 u'FullyConnected/W',
 u'FullyConnected/W/Initializer/truncated_normal/shape',
 u'FullyConnected/W/Initializer/truncated_normal/mean',
 u'FullyConnected/W/Initializer/truncated_normal/stddev',
 u'FullyConnected/W/Initializer/truncated_normal/TruncatedNormal',
 u'FullyConnected/W/Initializer/truncated_normal/mul',
 u'FullyConnected/W/Initializer/truncated_normal',
 u'FullyConnected/W/Assign',
 u'FullyConnected/W/read',
 u'FullyConnected/b',
 u'FullyConnected/b/Initializer/Const',
 u'FullyConnected/b/Assign',
 u'FullyConnected/b/read',
 u'FullyConnected/Reshape/shape',
 u'FullyConnected/Reshape',
 u'FullyConnected/MatMul',
 u'FullyConnected/BiasAdd',
 u'FullyConnected/Relu',
 u'is_training',
 u'is_training/Initializer/Const',
 u'is_training/Assign',
 u'is_training/read',
 u'Dropout/Assign/value',
 u'Dropout/Assign',
 u'Dropout/Assign_1/value',
 u'Dropout/Assign_1',
 u'Dropout/cond/Switch',
 u'Dropout/cond/switch_t',
 u'Dropout/cond/switch_f',
 u'Dropout/cond/pred_id',
 u'Dropout/cond/dropout/keep_prob',
 u'Dropout/cond/dropout/Shape/Switch',
 u'Dropout/cond/dropout/Shape',
 u'Dropout/cond/dropout/random_uniform/min',
 u'Dropout/cond/dropout/random_uniform/max',
 u'Dropout/cond/dropout/random_uniform/RandomUniform',
 u'Dropout/cond/dropout/random_uniform/sub',
 u'Dropout/cond/dropout/random_uniform/mul',
 u'Dropout/cond/dropout/random_uniform',
 u'Dropout/cond/dropout/add',
 u'Dropout/cond/dropout/Floor',
 u'Dropout/cond/dropout/Div',
 u'Dropout/cond/dropout/mul',
 u'Dropout/cond/Switch_1',
 u'Dropout/cond/Merge',
 u'output/W',
 u'output/W/Initializer/truncated_normal/shape',
 u'output/W/Initializer/truncated_normal/mean',
 u'output/W/Initializer/truncated_normal/stddev',
 u'output/W/Initializer/truncated_normal/TruncatedNormal',
 u'output/W/Initializer/truncated_normal/mul',
 u'output/W/Initializer/truncated_normal',
 u'output/W/Assign',
 u'output/W/read',
 u'output/b',
 u'output/b/Initializer/Const',
 u'output/b/Assign',
 u'output/b/read',
 u'output/MatMul',
 u'output/BiasAdd',
 u'output/Softmax',
 u'Log',
 u'mul',
 u'Sum/reduction_indices',
 u'Sum',
 u'Neg',
 u'Const',
 u'Mean',
 u'gradients/Shape',
 u'gradients/Const',
 u'gradients/Fill',
 u'gradients/Mean_grad/Reshape/shape',
 u'gradients/Mean_grad/Reshape',
 u'gradients/Mean_grad/Shape',
 u'gradients/Mean_grad/Tile',
 u'gradients/Mean_grad/Shape_1',
 u'gradients/Mean_grad/Shape_2',
 u'gradients/Mean_grad/Const',
 u'gradients/Mean_grad/Prod',
 u'gradients/Mean_grad/Const_1',
 u'gradients/Mean_grad/Prod_1',
 u'gradients/Mean_grad/Maximum/y',
 u'gradients/Mean_grad/Maximum',
 u'gradients/Mean_grad/floordiv',
 u'gradients/Mean_grad/Cast',
 u'gradients/Mean_grad/truediv',
 u'gradients/Neg_grad/Neg',
 u'gradients/Sum_grad/Shape',
 u'gradients/Sum_grad/Size',
 u'gradients/Sum_grad/add',
 u'gradients/Sum_grad/mod',
 u'gradients/Sum_grad/Shape_1',
 u'gradients/Sum_grad/range/start',
 u'gradients/Sum_grad/range/delta',
 u'gradients/Sum_grad/range',
 u'gradients/Sum_grad/Fill/value',
 u'gradients/Sum_grad/Fill',
 u'gradients/Sum_grad/DynamicStitch',
 u'gradients/Sum_grad/Maximum/y',
 u'gradients/Sum_grad/Maximum',
 u'gradients/Sum_grad/floordiv',
 u'gradients/Sum_grad/Reshape',
 u'gradients/Sum_grad/Tile',
 u'gradients/mul_grad/Shape',
 u'gradients/mul_grad/Shape_1',
 u'gradients/mul_grad/BroadcastGradientArgs',
 u'gradients/mul_grad/mul',
 u'gradients/mul_grad/Sum',
 u'gradients/mul_grad/Reshape',
 u'gradients/mul_grad/mul_1',
 u'gradients/mul_grad/Sum_1',
 u'gradients/mul_grad/Reshape_1',
 u'gradients/mul_grad/tuple/group_deps',
 u'gradients/mul_grad/tuple/control_dependency',
 u'gradients/mul_grad/tuple/control_dependency_1',
 u'gradients/Log_grad/Inv',
 u'gradients/Log_grad/mul',
 u'gradients/output/Softmax_grad/mul',
 u'gradients/output/Softmax_grad/Sum/reduction_indices',
 u'gradients/output/Softmax_grad/Sum',
 u'gradients/output/Softmax_grad/Reshape/shape',
 u'gradients/output/Softmax_grad/Reshape',
 u'gradients/output/Softmax_grad/sub',
 u'gradients/output/Softmax_grad/mul_1',
 u'gradients/output/BiasAdd_grad/BiasAddGrad',
 u'gradients/output/BiasAdd_grad/tuple/group_deps',
 u'gradients/output/BiasAdd_grad/tuple/control_dependency',
 u'gradients/output/BiasAdd_grad/tuple/control_dependency_1',
 u'gradients/output/MatMul_grad/MatMul',
 u'gradients/output/MatMul_grad/MatMul_1',
 u'gradients/output/MatMul_grad/tuple/group_deps',
 u'gradients/output/MatMul_grad/tuple/control_dependency',
 u'gradients/output/MatMul_grad/tuple/control_dependency_1',
 u'gradients/Dropout/cond/Merge_grad/cond_grad',
 u'gradients/Dropout/cond/Merge_grad/tuple/group_deps',
 u'gradients/Dropout/cond/Merge_grad/tuple/control_dependency',
 u'gradients/Dropout/cond/Merge_grad/tuple/control_dependency_1',
 u'gradients/Switch',
 u'gradients/Shape_1',
 u'gradients/zeros/Const',
 u'gradients/zeros',
 u'gradients/Dropout/cond/Switch_1_grad/cond_grad',
 u'gradients/Dropout/cond/dropout/mul_grad/Shape',
 u'gradients/Dropout/cond/dropout/mul_grad/Shape_1',
 u'gradients/Dropout/cond/dropout/mul_grad/BroadcastGradientArgs',
 u'gradients/Dropout/cond/dropout/mul_grad/mul',
 u'gradients/Dropout/cond/dropout/mul_grad/Sum',
 u'gradients/Dropout/cond/dropout/mul_grad/Reshape',
 u'gradients/Dropout/cond/dropout/mul_grad/mul_1',
 u'gradients/Dropout/cond/dropout/mul_grad/Sum_1',
 u'gradients/Dropout/cond/dropout/mul_grad/Reshape_1',
 u'gradients/Dropout/cond/dropout/mul_grad/tuple/group_deps',
 u'gradients/Dropout/cond/dropout/mul_grad/tuple/control_dependency',
 u'gradients/Dropout/cond/dropout/mul_grad/tuple/control_dependency_1',
 u'gradients/Dropout/cond/dropout/Div_grad/Shape',
 u'gradients/Dropout/cond/dropout/Div_grad/Shape_1',
 u'gradients/Dropout/cond/dropout/Div_grad/BroadcastGradientArgs',
 u'gradients/Dropout/cond/dropout/Div_grad/truediv',
 u'gradients/Dropout/cond/dropout/Div_grad/Sum',
 u'gradients/Dropout/cond/dropout/Div_grad/Reshape',
 u'gradients/Dropout/cond/dropout/Div_grad/Neg',
 u'gradients/Dropout/cond/dropout/Div_grad/Square',
 u'gradients/Dropout/cond/dropout/Div_grad/truediv_1',
 u'gradients/Dropout/cond/dropout/Div_grad/mul',
 u'gradients/Dropout/cond/dropout/Div_grad/Sum_1',
 u'gradients/Dropout/cond/dropout/Div_grad/Reshape_1',
 u'gradients/Dropout/cond/dropout/Div_grad/tuple/group_deps',
 u'gradients/Dropout/cond/dropout/Div_grad/tuple/control_dependency',
 u'gradients/Dropout/cond/dropout/Div_grad/tuple/control_dependency_1',
 u'gradients/Switch_1',
 u'gradients/Shape_2',
 u'gradients/zeros_1/Const',
 u'gradients/zeros_1',
 u'gradients/Dropout/cond/dropout/Shape/Switch_grad/cond_grad',
 u'gradients/AddN',
 u'gradients/FullyConnected/Relu_grad/ReluGrad',
 u'gradients/FullyConnected/BiasAdd_grad/BiasAddGrad',
 u'gradients/FullyConnected/BiasAdd_grad/tuple/group_deps',
 u'gradients/FullyConnected/BiasAdd_grad/tuple/control_dependency',
 u'gradients/FullyConnected/BiasAdd_grad/tuple/control_dependency_1',
 u'gradients/FullyConnected/MatMul_grad/MatMul',
 u'gradients/FullyConnected/MatMul_grad/MatMul_1',
 u'gradients/FullyConnected/MatMul_grad/tuple/group_deps',
 u'gradients/FullyConnected/MatMul_grad/tuple/control_dependency',
 u'gradients/FullyConnected/MatMul_grad/tuple/control_dependency_1',
 u'gradients/FullyConnected/Reshape_grad/Shape',
 u'gradients/FullyConnected/Reshape_grad/Reshape',
 u'gradients/MaxPool2D_1/MaxPool_grad/MaxPoolGrad',
 u'gradients/conv_layer_2/Relu_grad/ReluGrad',
 u'gradients/conv_layer_2/BiasAdd_grad/BiasAddGrad',
 u'gradients/conv_layer_2/BiasAdd_grad/tuple/group_deps',
 u'gradients/conv_layer_2/BiasAdd_grad/tuple/control_dependency',
 u'gradients/conv_layer_2/BiasAdd_grad/tuple/control_dependency_1',
 u'gradients/conv_layer_2/Conv2D_grad/Shape',
 u'gradients/conv_layer_2/Conv2D_grad/Conv2DBackpropInput',
 u'gradients/conv_layer_2/Conv2D_grad/Shape_1',
 u'gradients/conv_layer_2/Conv2D_grad/Conv2DBackpropFilter',
 u'gradients/conv_layer_2/Conv2D_grad/tuple/group_deps',
 u'gradients/conv_layer_2/Conv2D_grad/tuple/control_dependency',
 u'gradients/conv_layer_2/Conv2D_grad/tuple/control_dependency_1',
 u'gradients/MaxPool2D/MaxPool_grad/MaxPoolGrad',
 u'gradients/conv_layer_1/Relu_grad/ReluGrad',
 u'gradients/conv_layer_1/BiasAdd_grad/BiasAddGrad',
 u'gradients/conv_layer_1/BiasAdd_grad/tuple/group_deps',
 u'gradients/conv_layer_1/BiasAdd_grad/tuple/control_dependency',
 u'gradients/conv_layer_1/BiasAdd_grad/tuple/control_dependency_1',
 u'gradients/conv_layer_1/Conv2D_grad/Shape',
 u'gradients/conv_layer_1/Conv2D_grad/Conv2DBackpropInput',
 u'gradients/conv_layer_1/Conv2D_grad/Shape_1',
 u'gradients/conv_layer_1/Conv2D_grad/Conv2DBackpropFilter',
 u'gradients/conv_layer_1/Conv2D_grad/tuple/group_deps',
 u'gradients/conv_layer_1/Conv2D_grad/tuple/control_dependency',
 u'gradients/conv_layer_1/Conv2D_grad/tuple/control_dependency_1',
 u'beta1_power/initial_value',
 u'beta1_power',
 u'beta1_power/Assign',
 u'beta1_power/read',
 u'beta2_power/initial_value',
 u'beta2_power',
 u'beta2_power/Assign',
 u'beta2_power/read',
 u'zeros',
 u'conv_layer_1/W/Adam',
 u'conv_layer_1/W/Adam/Assign',
 u'conv_layer_1/W/Adam/read',
 u'zeros_1',
 u'conv_layer_1/W/Adam_1',
 u'conv_layer_1/W/Adam_1/Assign',
 u'conv_layer_1/W/Adam_1/read',
 u'zeros_2',
 u'conv_layer_1/b/Adam',
 u'conv_layer_1/b/Adam/Assign',
 u'conv_layer_1/b/Adam/read',
 u'zeros_3',
 u'conv_layer_1/b/Adam_1',
 u'conv_layer_1/b/Adam_1/Assign',
 u'conv_layer_1/b/Adam_1/read',
 u'zeros_4',
 u'conv_layer_2/W/Adam',
 u'conv_layer_2/W/Adam/Assign',
 u'conv_layer_2/W/Adam/read',
 u'zeros_5',
 u'conv_layer_2/W/Adam_1',
 u'conv_layer_2/W/Adam_1/Assign',
 u'conv_layer_2/W/Adam_1/read',
 u'zeros_6',
 u'conv_layer_2/b/Adam',
 u'conv_layer_2/b/Adam/Assign',
 u'conv_layer_2/b/Adam/read',
 u'zeros_7',
 u'conv_layer_2/b/Adam_1',
 u'conv_layer_2/b/Adam_1/Assign',
 u'conv_layer_2/b/Adam_1/read',
 u'zeros_8',
 u'FullyConnected/W/Adam',
 u'FullyConnected/W/Adam/Assign',
 u'FullyConnected/W/Adam/read',
 u'zeros_9',
 u'FullyConnected/W/Adam_1',
 u'FullyConnected/W/Adam_1/Assign',
 u'FullyConnected/W/Adam_1/read',
 u'zeros_10',
 u'FullyConnected/b/Adam',
 u'FullyConnected/b/Adam/Assign',
 u'FullyConnected/b/Adam/read',
 u'zeros_11',
 u'FullyConnected/b/Adam_1',
 u'FullyConnected/b/Adam_1/Assign',
 u'FullyConnected/b/Adam_1/read',
 u'zeros_12',
 u'output/W/Adam',
 u'output/W/Adam/Assign',
 u'output/W/Adam/read',
 u'zeros_13',
 u'output/W/Adam_1',
 u'output/W/Adam_1/Assign',
 u'output/W/Adam_1/read',
 u'zeros_14',
 u'output/b/Adam',
 u'output/b/Adam/Assign',
 u'output/b/Adam/read',
 u'zeros_15',
 u'output/b/Adam_1',
 u'output/b/Adam_1/Assign',
 u'output/b/Adam_1/read',
 u'Adam/learning_rate',
 u'Adam/beta1',
 u'Adam/beta2',
 u'Adam/epsilon',
 u'Adam/update_conv_layer_1/W/ApplyAdam',
 u'Adam/update_conv_layer_1/b/ApplyAdam',
 u'Adam/update_conv_layer_2/W/ApplyAdam',
 u'Adam/update_conv_layer_2/b/ApplyAdam',
 u'Adam/update_FullyConnected/W/ApplyAdam',
 u'Adam/update_FullyConnected/b/ApplyAdam',
 u'Adam/update_output/W/ApplyAdam',
 u'Adam/update_output/b/ApplyAdam',
 u'Adam/mul',
 u'Adam/Assign',
 u'Adam/mul_1',
 u'Adam/Assign_1',
 u'Adam',
 u'ArgMax/dimension',
 u'ArgMax',
 u'ArgMax_1/dimension',
 u'ArgMax_1',
 u'Equal',
 u'Cast',
 u'Const_1',
 u'Mean_1',
 u'init']



In [15]:

    
#number of interations
epoch=15000
batch_size=50



In [16]:

    
for i in range(epoch):
    #batch wise training 
    x_batch, y_batch = data.train.next_batch(batch_size)
    _,loss=sess.run([train_step, cross_entropy], feed_dict={x: x_batch,y_: y_batch})
    #_, loss,acc=sess.run([train_step,cross_entropy,accuracy], feed_dict={x:input_image , y_: input_class})
    
    if i%500==0:    
        Accuracy=sess.run(accuracy,
                           feed_dict={
                        x: data.test.images,
                        y_: data.test.labels
                      })
        Accuracy=round(Accuracy*100,2)
        print "Loss : {} , Accuracy on test set : {} %" .format(loss, Accuracy)
    elif i%100==0:
        print "Loss : {}" .format(loss)









    



Loss : 2.28293037415 , Accuracy on test set : 11.18 %
Loss : 0.597587764263
Loss : 0.287460297346
Loss : 0.386544346809
Loss : 0.128280937672
Loss : 0.251751989126 , Accuracy on test set : 94.31 %
Loss : 0.0702344104648
Loss : 0.167915225029
Loss : 0.22772346437
Loss : 0.0516111552715
Loss : 0.0974466353655 , Accuracy on test set : 96.85 %
Loss : 0.177406698465
Loss : 0.0460326112807
Loss : 0.162814065814
Loss : 0.0267831366509
Loss : 0.0413384996355 , Accuracy on test set : 97.63 %
Loss : 0.0990437492728
Loss : 0.136322826147
Loss : 0.098825648427
Loss : 0.0265418570489
Loss : 0.030837232247 , Accuracy on test set : 98.04 %
Loss : 0.0933626368642
Loss : 0.0128259947523
Loss : 0.0305090136826
Loss : 0.0323739536107
Loss : 0.220340594649 , Accuracy on test set : 98.14 %
Loss : 0.129726827145
Loss : 0.0139974402264
Loss : 0.0734559446573
Loss : 0.0500608049333
Loss : 0.126619532704 , Accuracy on test set : 98.5 %
Loss : 0.0393156372011
Loss : 0.0593934878707
Loss : 0.149956151843
Loss : 0.080586835742
Loss : 0.0419804044068 , Accuracy on test set : 98.58 %
Loss : 0.0819581896067
Loss : 0.0914292633533
Loss : 0.0670996382833
Loss : 0.0301091596484
Loss : 0.0144080091268 , Accuracy on test set : 98.67 %
Loss : 0.0526999533176
Loss : 0.0191994123161
Loss : 0.13336405158
Loss : 0.0193726792932
Loss : 0.0332157276571 , Accuracy on test set : 98.59 %
Loss : 0.0310003012419
Loss : 0.00332154217176
Loss : 0.041251540184
Loss : 0.00415246980265
Loss : 0.0432712472975 , Accuracy on test set : 99.03 %
Loss : 0.00304029812105
Loss : 0.00863337796181
Loss : 0.00990332663059
Loss : 0.00602502841502
Loss : 0.00429784506559 , Accuracy on test set : 98.96 %
Loss : 0.00483903940767
Loss : 0.02215398103
Loss : 0.00401343638077
Loss : 0.0194915216416
Loss : 0.0405216962099 , Accuracy on test set : 99.01 %
Loss : 0.0116603346542
Loss : 0.00443902425468
Loss : 0.0155848907307
Loss : 0.0736524164677
Loss : 0.00141466234345 , Accuracy on test set : 98.87 %
Loss : 0.0275035686791
Loss : 0.0507901348174
Loss : 0.008460175246
Loss : 0.00200296472758
Loss : 0.00874637812376 , Accuracy on test set : 99.08 %
Loss : 0.0951781868935
Loss : 0.00243265810423
Loss : 0.0174704845995
Loss : 0.0159406103194
Loss : 0.00320935994387 , Accuracy on test set : 98.82 %
Loss : 0.000968973268755
Loss : 0.000859417836182
Loss : 0.00189435307402
Loss : 0.0143010048196
Loss : 0.100800283253 , Accuracy on test set : 99.05 %
Loss : 0.0168285053223
Loss : 0.0243952032179
Loss : 0.00551780173555
Loss : 0.00521994475275
Loss : 0.00650927703828 , Accuracy on test set : 99.09 %
Loss : 0.010179891251
Loss : 0.00123321264982
Loss : 0.0405683293939
Loss : 0.00396152865142
Loss : 0.0192007832229 , Accuracy on test set : 99.12 %
Loss : 0.00875569880009
Loss : 0.0052846991457
Loss : 0.0532456301153
Loss : 0.0045698704198
Loss : 0.00953463278711 , Accuracy on test set : 99.07 %
Loss : 0.00331870373338
Loss : 0.0478585064411
Loss : 0.00122305331752
Loss : 0.00665632542223
Loss : 0.00618476327509 , Accuracy on test set : 99.16 %
Loss : 0.000889086863026
Loss : 0.00213621859439
Loss : 0.00211060652509
Loss : 0.0155455758795
Loss : 0.0025133532472 , Accuracy on test set : 99.12 %
Loss : 0.00357839581557
Loss : 0.0649118348956
Loss : 0.0103517081589
Loss : 0.315198481083
Loss : 0.0191255751997 , Accuracy on test set : 99.17 %
Loss : 0.000846715644002
Loss : 0.00209143664688
Loss : 0.0022159346845
Loss : 0.00452533178031
Loss : 0.0768144801259 , Accuracy on test set : 99.11 %
Loss : 0.000642220024019
Loss : 0.0252051111311
Loss : 0.000853506149724
Loss : 0.000386520812754
Loss : 0.0169414598495 , Accuracy on test set : 99.21 %
Loss : 0.0254887659103
Loss : 0.0061031319201
Loss : 0.0206306837499
Loss : 0.00244550639763
Loss : 0.00687860418111 , Accuracy on test set : 99.18 %
Loss : 0.00236043799669
Loss : 0.00345431175083
Loss : 0.00429856078699
Loss : 0.00244203023612
Loss : 0.00314609147608 , Accuracy on test set : 99.1 %
Loss : 0.000838458363432
Loss : 0.00133332493715
Loss : 0.000294020952424
Loss : 0.00598060246557
Loss : 0.0064719892107 , Accuracy on test set : 99.14 %
Loss : 0.00134585204069
Loss : 0.000875915284269
Loss : 0.00221746531315
Loss : 0.000938961573411
Loss : 0.00147439201828 , Accuracy on test set : 98.9 %
Loss : 0.00257780542597
Loss : 0.000907330890186
Loss : 0.00188101967797
Loss : 0.00597016233951
Loss : 0.000222059024964 , Accuracy on test set : 98.97 %
Loss : 0.000812933722045
Loss : 0.00148851121776
Loss : 0.000600711966399
Loss : 0.00163220148534



In [17]:

    
validation_accuracy=round((sess.run(accuracy,
                            feed_dict={
                             x: data.validation.images,
                             y_: data.validation.labels
                              }))*100,2)

print "Accuracy in the validation dataset: {}%".format(validation_accuracy)









    



Accuracy in the validation dataset: 99.14%



In [18]:

    
#testset predictions
y_test=(sess.run(y_predicted,feed_dict={
                             x: data.test.images
                              }))



In [19]:

    
#Confusion Matrix
true_class=np.argmax(data.test.labels,1)
predicted_class=np.argmax(y_test,1)
cm=confusion_matrix(predicted_class,true_class)
cm









    Out[19]:





array([[ 976,    0,    0,    0,    0,    2,    3,    0,    3,    0],
       [   1, 1134,    1,    0,    0,    0,    2,    2,    0,    0],
       [   0,    0, 1029,    2,    0,    0,    0,    7,    2,    0],
       [   0,    0,    0, 1002,    0,    7,    1,    1,    2,    1],
       [   0,    0,    0,    0,  976,    0,    4,    0,    0,    3],
       [   0,    0,    0,    2,    0,  879,    4,    0,    1,    1],
       [   1,    0,    0,    0,    1,    1,  944,    0,    0,    0],
       [   0,    1,    2,    0,    0,    0,    0, 1012,    2,    0],
       [   1,    0,    0,    1,    0,    1,    0,    1,  958,    1],
       [   1,    0,    0,    3,    5,    2,    0,    5,    6, 1003]])



In [20]:

    
#Plotting confusion Matrix
plt.imshow(cm,interpolation='nearest')
plt.colorbar()
number_of_class=len(np.unique(true_class))
tick_marks = np.arange(len(np.unique(true_class)))
plt.xticks(tick_marks, range(number_of_class))
plt.yticks(tick_marks, range(number_of_class))
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.title('Confusion Matrix')
plt.show()



In [21]:

    
#Finding error outputs
idx=np.argmax(y_test,1)==np.argmax(data.test.labels,1) 
cmp=np.where(idx==False) #indices of error outputs
# plotting errors
fig, axes = plt.subplots(5, 3, figsize=(15,15))
fig.subplots_adjust(hspace=0.3, wspace=0.3)
cls_true=np.argmax(data.test.labels,1)[cmp]
cls_pred=np.argmax(y_test,1)[cmp]
images=data.test.images[cmp]
for i, ax in enumerate(axes.flat):
        ax.imshow(images[i].reshape(28,28), cmap='binary')
        xlabel = "True: {0}, Pred: {1}".format(cls_true[i], cls_pred[i])
        ax.set_xlabel(xlabel)
        ax.set_xticks([])
        ax.set_yticks([])      
plt.show()



In [22]:

    
conv_layer1_filters=conv_layer1.W.eval()
print conv_layer1_filters.shape









    



(5, 5, 1, 32)



In [23]:

    
conv_layer1_filters_img=conv_layer1_filters[:,:,0,:]
print conv_layer1_filters_img.shape









    



(5, 5, 32)



In [24]:

    
#plotting filters of the first convolutional layer
fig, axes = plt.subplots(8, 4, figsize=(15,15))
fig.subplots_adjust(hspace=0.3, wspace=0.3)
for i, ax in enumerate(axes.flat):
        ax.imshow(conv_layer1_filters_img[:,:,i], cmap='gray')
        xlabel = "Filter : {}".format(i+1)
        ax.set_xlabel(xlabel)
        ax.set_xticks([])
        ax.set_yticks([]) 
plt.show()



In [25]:

    
test_image=np.reshape(data.test.images[0], [1,784])
conv_layer1_output=(sess.run(out_layer1,
               feed_dict={
                   x:test_image
               }))



In [26]:

    
plt.imshow(np.reshape(data.test.images[0], [28,28]), cmap='gray')
plt.title('Test Image')
plt.axis('off')
plt.show()



In [27]:

    
print conv_layer1_output.shape









    



(1, 14, 14, 32)



In [28]:

    
conv_layer1_output_img=conv_layer1_output[0,:,:,:]
fig, axes = plt.subplots(8, 4, figsize=(15,15))
fig.subplots_adjust(hspace=0.3, wspace=0.3)

for i, ax in enumerate(axes.flat):
        ax.imshow(conv_layer1_output_img[:,:,i], cmap='gray')
        xlabel = "Filter : {}".format(i+1)
        ax.set_xlabel(xlabel)
        ax.set_xticks([])
        ax.set_yticks([])      
fig.suptitle('Output of the first convolutional layer')  
plt.show()

Testing your own handwritten digits



In [29]:

    
im=Image.open("/Users/Enkay/Documents/Viky/DL-channel/MNIST/images/s4.jpg")
im









    Out[29]:



In [30]:

    
im=im.resize((28, 28), Image.ANTIALIAS) #resize the image
im = np.array(im) #convert to an array
im2=im/np.max(im).astype(float) #normalise input
test_image1=np.reshape(im2, [1,784]) # reshape it to our input placeholder shape



In [31]:

    
pred=(sess.run(y_predicted,
               feed_dict={
                   x:test_image1
               }))
predicted_class=np.argmax(pred)
print "Predicted class : {}" .format(predicted_class)









    



Predicted class : 7



In [32]:

    
five=Image.open("/Users/Enkay/Documents/Viky/DL-channel/MNIST/images/five.jpeg")
five









    Out[32]:



In [33]:

    
five=five.resize((28, 28), Image.ANTIALIAS) #resize the image
five = np.array(five)
five_test=five/np.max(five).astype(float) 
five_test=np.reshape(five_test, [1,784])



In [36]:

    
pred=(sess.run(y_predicted,
               feed_dict={
                   x:five_test
               }))
predicted_class=np.argmax(pred)
print "Predicted class : {}" .format(predicted_class)









    



Predicted class : 5



In [ ]: